<?php 
class Extractor
{
    /**
     * Store indexation uri
     * @var string
     */
    public $url;
    /**
     * Store dom of current pgae
     * @var object
     */
    private $dom;
    /**
     * Store object of current curl 
     * @var object
     */
    private $curl;        
    /**
     * Variable with response headers
     * @var array
     */
    private $headers;
    /**
     * Variable with server result
     * @var string
     */
    private $result;
    
    
    function __construct ()
    {
        $this->curl = curl_init();
    }
    
    
    public function getDomain ($url = null)
    {
      $parsed_url = parse_url(urldecode($url));

      if(is_array($parsed_url) && count($parsed_url))
        if(isset($parsed_url["host"]))
          return $parsed_url["host"];      
      return null;
    }

    
    
    public function doNormalizeLink ($parent, $link)
    {
      
        $parsed_parent   = parse_url(urldecode($parent));
        $parsed_link     = parse_url(urldecode($link));
      
        if(! $parsed_parent || ! $parsed_link)
          return null;
                  
        $parent_keys_to_remove = array(
        	"user"     => "", 
        	"pass"     => "", 
        	"path"     => "", 
        	"query"    => "", 
        	"fragment" => ""
        );
        
        $parsed_parent   = array_diff_key($parsed_parent, $parent_keys_to_remove);
        
        $link_keys_to_remove = array(
          "user"     => "", 
          "pass"     => "", 
          "fragment" => ""
        );        
        
        $parsed_link     = array_diff_key($parsed_link, $link_keys_to_remove);
        
        if(! $parsed_parent || ! $parsed_link)
          return null;
        
        return urldecode(http_build_url(array_merge($parsed_parent, $parsed_link)));
    }
    /**
     * Check is link is inner
     * @param string $parent
     * @param string $link
     */
    public function isInnerLink ($parent, $link)
    {
        $parsed_parent   = parse_url($parent);
        $parsed_link     = parse_url($link);
        
        if(! isset($parsed_link["host"]))
            return true;
        if($parsed_parent["host"] == $parsed_link["host"])
            if(isset($parsed_parent["scheme"]) && isset($parsed_link["scheme"]))
                if($parsed_parent["scheme"] == $parsed_link["scheme"])
                    return true;
        return false;
    }
    
    /**
     * Method to get inner links
     * @param $url
     */
    public function getInnerLinks ($url)
    {
        $xpath = new DOMXPath($this->getContent($url));
        
        $hrefs = $xpath->evaluate("//a"); //get all a tags
        
        $links = array();
        for($i = 0; $i < $hrefs->length; $i ++) {
            $href = $hrefs->item($i); //select an a tag
            $link = $this->doNormalizeLink($url, $href->getAttribute('href'));
            if(! $this->isInnerLink($url, $link))
                continue;
            $links[] = $link;
        }
        return $links;
    }
    /**
     * Method to get outer links
     * @param $url
     */    
    public function getOuterLinks ($url)
    {
        $xpath = new DOMXPath($this->getContent($url));
                
        $hrefs = $xpath->evaluate("//a"); //get all a tags
        $links = array();
        for($i = 0; $i < $hrefs->length; $i ++) {
            $href = $hrefs->item($i); //select an a tag
            $link = $this->doNormalizeLink($url, $href->getAttribute('href'));
            if($this->isInnerLink($url, $link))
                continue;
            $links[] = $link;
        }
        return $links;
    }
//    public function doProcessPage ($url)
//    {
//        $xpath = new DOMXPath($this->getContent($url));
//        
//        $hrefs = $xpath->evaluate("//a"); //get all a tags
//        $links = array();
//        for($i = 0; $i < $hrefs->length; $i ++) {
//            $href = $hrefs->item($i); //select an a tag
//            $links[] = $this->doNormalizeLink ($url, $href->getAttribute('href'));
//        }
//        return $links;
//    }        
    /**
     * Check for permanently moved page
     * @return boolean
     */
    public function isMovedPermanently ()
    {
      $statuses = array("301", "302");
      
      foreach ($statuses as $status)
      {
        if(strpos($this->result, $status) !== false)
          return true;
      }
      return false;
    }    
    /**
     * Method to get needed header value
     * @param $header
     */
    public function getHeader ($header)
    {
      
      if(!isset($this->headers))
        throw new Exception ("Request for unknown header!");
        
      return $this->headers[$header];
    }
    
    
    /**
     *  Read headers and store to common array 
     * @param resourse $curl
     * @param string $header
     */
    protected function read_header($curl, $header)
    {

      $header_length = mb_strlen($header);
      $header        = trim ($header);
      
      if(mb_strlen (trim($header)))
      {
        $delimiter_position = strpos($header, ":");
        
        if($delimiter_position === false)
        {
          $this->result = $header;
          return $header_length;
        }
        
        $key    = trim(substr($header, 0, $delimiter_position));
        $value  = trim(substr($header, $delimiter_position + 1));
        
        $this->headers[$key] = $value;
      }
      
      return $header_length;
    } 

    
    public function setHeaders ($array = array ())
    {
      $this->headers = $array;
    }
    
    function getContent ($url)
    {
        if($this->url == $url)
            if(is_object($this->dom))
                return $this->dom;
                
        $this->url                            = $url;
        
        $options_curl                         = array ();
        $options_curl[CURLOPT_URL]            = $url;
        $options_curl[CURLOPT_RETURNTRANSFER] = true;
        $options_curl[CURLOPT_HEADERFUNCTION] = array ($this, 'read_header');
        $options_curl[CURLOPT_USERAGENT]      = 'Googlebot/2.1 (+http://www.google.com/bot.html)';
        $options_curl[CURLOPT_HTTPHEADER]     = $this->headers;
        
        curl_setopt_array ($this->curl, $options_curl);
        
        $this->content_raw   = curl_exec($this->curl);
        
        if(!mb_strlen ($this->content_raw))
        {
          if($this->isMovedPermanently ())
          {
            $options_curl[CURLOPT_URL] = $this->getHeader("Location");
            
            curl_setopt_array ($this->curl, $options_curl);

            $this->content_raw   = curl_exec($this->curl);
          }
        }

        file_put_contents ("content.txt", $this->content_raw);
        
        $this->dom = new DOMDocument();
        
//        $tidy      = new tidy();        
//        @$this->dom->loadHTML($tidy->repairString($content));

        @$this->dom->loadHTML($this->content_raw);
        
        return $this->dom;
    }
}


?>